去除Sigmoid函數，因為它會在極端狀況下導致近似值計算失準。
無論Discriminator跟Generator誰佔優勢，選擇一個平滑的損失函數來描述目前的競賽狀況。
註：數值越小代表越好！
\[ \begin{align} loss(y, x) & = (1-y)x - yx \end{align} \]
這個函數的意義是，當\(y=0\)時，那他希望\(x \rightarrow -\infty\)，而\(y=1\)時，那他希望\(x \rightarrow \infty\)。
除此之外，他還對Discriminator的Weight做了一個特殊的操作，那就是限制Discriminator中所有的Weight不超過某個數(一般訂為0.1)。
set.seed(0)
n.epoch <- 20
w_limit <- 0.1
logger <- list(gen_loss = NULL, dis_real_loss = NULL, dis_fake_loss = NULL)for (j in 1:n.epoch) {
current_batch <- 0
my_iter$reset()
while (my_iter$iter.next()) {
my_values <- my_iter$value()
# Generator (forward)
mx.exec.update.arg.arrays(gen_executor, arg.arrays = list(data = my_values[['noise']], digit = my_values[['digit.fake']]), match.name = TRUE)
mx.exec.forward(gen_executor, is.train = TRUE)
gen_pred_output <- gen_executor$ref.outputs[[1]]
# Discriminator (fake)
mx.exec.update.arg.arrays(dis_executor, arg.arrays = list(img = gen_pred_output, digit = my_values[['digit.fake']], label = my_values[['label.fake']]), match.name = TRUE)
mx.exec.forward(dis_executor, is.train = TRUE)
mx.exec.backward(dis_executor)
dis_update_args <- dis_updater(weight = dis_executor$ref.arg.arrays, grad = dis_executor$ref.grad.arrays)
mx.exec.update.arg.arrays(dis_executor, dis_update_args, skip.null = TRUE)
logger$dis_fake_loss <- c(logger$dis_fake_loss, as.array(dis_executor$ref.outputs[[1]]))
# Discriminator (real)
mx.exec.update.arg.arrays(dis_executor, arg.arrays = list(img = my_values[['img']], digit = my_values[['digit.real']], label = my_values[['label.real']]), match.name = TRUE)
mx.exec.forward(dis_executor, is.train = TRUE)
mx.exec.backward(dis_executor)
dis_update_args <- dis_updater(weight = dis_executor$ref.arg.arrays, grad = dis_executor$ref.grad.arrays)
mx.exec.update.arg.arrays(dis_executor, dis_update_args, skip.null = TRUE)
logger$dis_real_loss <- c(logger$dis_real_loss, as.array(dis_executor$ref.outputs[[1]]))
# Weight clipping (only for discriminator)
dis_weight_names <- grep('weight', names(dis_executor$ref.arg.arrays), value = TRUE)
for (k in dis_weight_names) {
current_dis_weight <- dis_executor$ref.arg.arrays[[k]] %>% as.array()
current_dis_weight_list <- current_dis_weight %>% mx.nd.array() %>%
mx.nd.broadcast.minimum(., mx.nd.array(w_limit)) %>%
mx.nd.broadcast.maximum(., mx.nd.array(-w_limit)) %>%
list()
names(current_dis_weight_list) <- k
mx.exec.update.arg.arrays(dis_executor, arg.arrays = current_dis_weight_list, match.name = TRUE)
}
# Generator (backward)
mx.exec.update.arg.arrays(dis_executor, arg.arrays = list(img = gen_pred_output, digit = my_values[['digit.fake']], label = my_values[['label.gen']]), match.name = TRUE)
mx.exec.forward(dis_executor, is.train = TRUE)
mx.exec.backward(dis_executor)
img_grads <- dis_executor$ref.grad.arrays[['img']]
mx.exec.backward(gen_executor, out_grads = img_grads)
gen_update_args <- gen_updater(weight = gen_executor$ref.arg.arrays, grad = gen_executor$ref.grad.arrays)
mx.exec.update.arg.arrays(gen_executor, gen_update_args, skip.null = TRUE)
logger$gen_loss <- c(logger$gen_loss, as.array(dis_executor$ref.outputs[[1]]))
if (current_batch %% 100 == 0) {
# Show current images
current_digits <- my_values[['digit.fake']] %>% as.array() %>% .[,,,1:9] %>% t %>% max.col - 1
par(mfrow = c(3, 3), mar = c(0.1, 0.1, 0.1, 0.1))
for (i in 1:9) {
img <- as.array(gen_pred_output)[,,,i]
plot(NA, xlim = 0:1, ylim = 0:1, xaxt = "n", yaxt = "n", bty = "n")
rasterImage(as.raster(t(img)), -0.04, -0.04, 1.04, 1.04, interpolate = FALSE)
text(0.05, 0.95, current_digits[i], col = 'green', cex = 2)
}
# Show loss
message('Epoch [', j, '] Batch [', current_batch, '] Generator-loss = ', formatC(tail(logger$gen_loss, 1), digits = 5, format = 'f'))
message('Epoch [', j, '] Batch [', current_batch, '] Discriminator-loss (real) = ', formatC(tail(logger$dis_real_loss, 1), digits = 5, format = 'f'))
message('Epoch [', j, '] Batch [', current_batch, '] Discriminator-loss (fake) = ', formatC(tail(logger$dis_fake_loss, 1), digits = 5, format = 'f'))
}
current_batch <- current_batch + 1
}
pdf(paste0('result/epoch_', j, '.pdf'), height = 6, width = 6)
current_digits <- my_values[['digit.fake']] %>% as.array() %>% .[,,,1:9] %>% t %>% max.col - 1
par(mfrow = c(3, 3), mar = c(0.1, 0.1, 0.1, 0.1))
for (i in 1:9) {
img <- as.array(gen_pred_output)[,,,i]
plot(NA, xlim = 0:1, ylim = 0:1, xaxt = "n", yaxt = "n", bty = "n")
rasterImage(as.raster(t(img)), -0.04, -0.04, 1.04, 1.04, interpolate = FALSE)
text(0.05, 0.95, current_digits[i], col = 'green', cex = 2)
}
dev.off()
gen_model <- list()
gen_model$symbol <- gen_pred
gen_model$arg.params <- gen_executor$ref.arg.arrays[-c(1:2)]
gen_model$aux.params <- gen_executor$ref.aux.arrays
class(gen_model) <- "MXFeedForwardModel"
dis_model <- list()
dis_model$symbol <- dis_pred
dis_model$arg.params <- dis_executor$ref.arg.arrays[-c(1:2)]
dis_model$aux.params <- dis_executor$ref.aux.arrays
class(dis_model) <- "MXFeedForwardModel"
mx.model.save(model = gen_model, prefix = 'model/cwgen_v1', iteration = j)
mx.model.save(model = dis_model, prefix = 'model/cwdis_v1', iteration = j)
}
range_logger <- logger %>% unlist %>% range
plot(logger$gen_loss, type = 'l', col = 'red', lwd = 0.5, ylim = range_logger, xlab = 'Batch', ylab = 'loss')
lines(1:length(logger$dis_real_loss), logger$dis_real_loss, col = 'blue', lwd = 0.5)
lines(1:length(logger$dis_fake_loss), logger$dis_fake_loss, col = 'darkgreen', lwd = 0.5)
legend('topright', c('Gen', 'Real', 'Fake'), col = c('red', 'blue', 'darkgreen'), lwd = 1)
cwgen_model <- mx.model.load('model/cwgen_v1', 0)
my_predict <- function (model, digits = 0:9) {
batch_size <- length(digits)
gen_executor <- mx.simple.bind(symbol = model$symbol,
data = c(1, 1, 10, batch_size), digit = c(1, 1, 10, batch_size),
ctx = mx.cpu())
mx.exec.update.arg.arrays(gen_executor, model$arg.params, match.name = TRUE)
mx.exec.update.aux.arrays(gen_executor, model$aux.params, match.name = TRUE)
noise_array <- rnorm(batch_size * 10, mean = 0, sd = 1)
noise_array <- array(noise_array, dim = c(1, 1, 10, batch_size))
noise_array <- mx.nd.array(noise_array)
digit_array <- mx.nd.array(digits)
digit_array <- mx.nd.one.hot(indices = digit_array, depth = 10)
digit_array <- mx.nd.reshape(data = digit_array, shape = c(1, 1, -1, batch_size))
mx.exec.update.arg.arrays(gen_executor, arg.arrays = list(data = noise_array, digit = digit_array), match.name = TRUE)
mx.exec.forward(gen_executor, is.train = FALSE)
gen_pred_output <- gen_executor$ref.outputs[[1]]
return(as.array(gen_pred_output))
}pred_img <- my_predict(model = cwgen_model, digits = 0:9)
par(mfrow = c(2, 5), mar = c(0.1, 0.1, 0.1, 0.1))
for (i in 1:10) {
img <- pred_img[,,,i]
plot(NA, xlim = 0:1, ylim = 0:1, xaxt = "n", yaxt = "n", bty = "n")
rasterImage(as.raster(t(img)), -0.04, -0.04, 1.04, 1.04, interpolate = FALSE)
}
– 這是Jun-Yan Zhu、Taesung Park、Phillip Isola與Alexei A. Efros在2017年提出的研究：Unpaired Image-to-Image Translation using Cycle-Consistent Adversarial Networks中所提出的模型
– 但比較厲害的是，Cycle GAN所輸入的條件並非是類似於Conditional GAN的結構化條件，而是直接利用一張圖片當作Condition。
– 在許多領域中，我們不可能蒐集到Paired data，舉例來說像是照片與藝術畫的轉換，因此這件事情是非常重要的！除此之外，Paired data的蒐集難度也高上非常多，這也是Cycle GAN所吸引人的地方所在。
– 假定有兩個函數分別是\(G(X)\)負責將\(X\)轉換為\(\hat{Y}\)，而另一個函數\(F(Y)\)負責將\(Y\)轉換為\(\hat{X}\)，則當兩個函數達到完美狀態時，必須保證\(F(G(X)) = X\)且\(G(F(Y)) = Y\)。
\[ \begin{align} \mbox{cycle consistency loss} & = |F(G(X)) - X| + |G(F(Y)) - Y| \end{align} \]
註：原始論文使用的是L1 loss(如上式)，而替換成L2 loss或者是其他損失函數影響不大。
– 這時候我們要引入兩個Discriminator分別是\(D_x(X)\)以及\(D_y(Y)\)，他們分別要識別翻譯出來的圖是真的還是假的，而在Generator跟Discriminator的競合我們將給出一個對抗損失(adversarial loss)，這與之前的所有GAN完全一樣，我們當然也能用WGAN的損失函數：
\[ \begin{align} \mbox{adversarial loss for Discriminator(x)} & = D_x(F(Y)) - D_x(X) \\ \mbox{adversarial loss for Discriminator(y)} & = D_y(G(X)) - D_y(Y) \\ \mbox{adversarial loss for Generator(x)} & = - D_x(F(Y)) \\ \mbox{adversarial loss for Generator(y)} & = - D_y(G(X)) \end{align} \]
– 這是原始照片：
library(OpenImageR)
library(jpeg)
photo <- readJPEG('images/header.jpg')
resize_photo <- resizeImage(image = photo,
width = 648,
height = 256,
method = "bilinear")
Show_img <- function (img) {
par(mai = rep(0, 4))
plot(NA, xlim = c(0.04, 0.96), ylim = c(0.96, 0.04), xaxt = "n", yaxt = "n", bty = "n")
rasterImage(as.raster(img), 0, 1, 1, 0, interpolate = FALSE)
}
Show_img(resize_photo)
library(mxnet)
P2M_gen_model <- mx.model.load(prefix = 'model/P2M_gen_v1', iteration = 0)
my_predict <- function (model, img) {
dim(img) <- c(dim(img), 1)
P2M_executor <- mx.simple.bind(symbol = model$symbol,
P2M_img = dim(img),
ctx = mx.cpu())
mx.exec.update.arg.arrays(P2M_executor, model$arg.params, match.name = TRUE)
mx.exec.update.aux.arrays(P2M_executor, model$aux.params, match.name = TRUE)
mx.exec.update.arg.arrays(P2M_executor, arg.arrays = list(P2M_img = mx.nd.array(img)), match.name = TRUE)
mx.exec.forward(P2M_executor, is.train = FALSE)
P2M_pred_output <- P2M_executor$ref.outputs[[1]]
return(as.array(P2M_pred_output)[,,,1])
}
monet_img <- my_predict(model = P2M_gen_model, img = resize_photo)
Show_img(monet_img)
– 為了解決這個問題，我們還需要額外發展出一個identity mapping loss來解決這個問題！
– 我們依此定義identity mapping loss如下(這邊要注意的是，函數\(G\)原來是負責\(X \rightarrow Y\)的，而函數\(F\)原來則是負責\(Y \rightarrow X\))：
\[ \begin{align} \mbox{identity mapping loss} & = |F(X) - X| + |G(Y) - Y| \end{align} \]
P2M_gen_model <- mx.model.load(prefix = 'model/P2M_gen_v2', iteration = 0)
monet_img <- my_predict(model = P2M_gen_model, img = resize_photo)
Show_img(monet_img)
– 讓我們看看效果吧！這是再過一次的效果(共計2次)：
monet_img <- my_predict(model = P2M_gen_model, img = monet_img)
Show_img(monet_img)
– 再過一次試試看(共計3次)：
monet_img <- my_predict(model = P2M_gen_model, img = monet_img)
Show_img(monet_img)
– 其實不用特別看大概也知道，由於Input size等於Output size，所以它的結構會與 前做Segmentation時的網路類似。
# Load data
load('data/mini_train_list.RData')
# Iterator function
my_iterator_core <- function (batch_size) {
batch <- 0
batch_per_epoch <- floor(length(train_list[[2]])/batch_size)
reset <- function() {batch <<- 0}
iter.next <- function() {
batch <<- batch + 1
if (batch > batch_per_epoch) {return(FALSE)} else {return(TRUE)}
}
value <- function() {
idx <- 1:batch_size + (batch - 1) * batch_size
img_array.1 <- array(0, dim = c(64, 64, 3, batch_size))
img_array.2 <- array(0, dim = c(64, 64, 3, batch_size))
for (i in 1:batch_size) {
img_array.1[,,,i] <- readJPEG(train_list[[1]][[idx[i]]])
img_array.2[,,,i] <- readJPEG(train_list[[2]][[idx[i]]])
}
img_array.1 <- mx.nd.array(img_array.1)
img_array.2 <- mx.nd.array(img_array.2)
return(list(monet = img_array.1, photo = img_array.2))
}
return(list(reset = reset, iter.next = iter.next, value = value, batch_size = batch_size, batch = batch))
}
my_iterator_func <- setRefClass("Custom_Iter",
fields = c("iter", "batch_size"),
contains = "Rcpp_MXArrayDataIter",
methods = list(
initialize = function(iter, batch_size = 16){
.self$iter <- my_iterator_core(batch_size = batch_size)
.self
},
value = function(){
.self$iter$value()
},
iter.next = function(){
.self$iter$iter.next()
},
reset = function(){
.self$iter$reset()
},
finalize=function(){
}
)
)
# Build an iterator
my_iter <- my_iterator_func(iter = NULL, batch_size = Batch_size)# Show image function
Show_img <- function (img) {
plot(NA, xlim = c(0.04, 0.96), ylim = c(0.96, 0.04), xaxt = "n", yaxt = "n", bty = "n")
rasterImage(as.raster(img), 0, 1, 1, 0, interpolate = FALSE)
}
# Test the iterator
my_iter$reset()
my_iter$iter.next()## [1] TRUEtest_data <- my_iter$value()
par(mai = rep(0, 4))
Show_img(as.array(test_data[[1]])[,,,1])
– 當然，想要真的訓練的很好，你可能需要加深加寬Model architecture，以及使用完整的資料集，而這樣需要極大的運算資源，你可能需要使用GPU server。
